package exp;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.List;

import tools.ObjectTools;
import tools.twitter.URLTools;
import tools.webkit.DownloadURL;
import tools.webkit.Webpage;

/**
 * Get the realURLs
 * @author Lanjun
 *
 */
public class EXP3_2 {
	static String filepath = "data/objs/urlContent/9_11_memorial_2011.dat.dl1";
	static DownloadURL d = new DownloadURL();
	
	public static void main(String[] args) {
		@SuppressWarnings("unchecked")
		List<Webpage> webpages = (ArrayList<Webpage>)ObjectTools.readFromFile(filepath);
		try {
			System.setOut(new PrintStream(new File("./logs/EXP3_2.out.log")));
			System.setErr(new PrintStream(new File("./logs/EXP3_2.err.log")));
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		}
		
		for (Webpage p : webpages) {
			p = urlResolution(p);
			System.out.println(p.url);
			System.out.println("\t" + p.realURL);
		}
		
		ObjectTools.writeToFile(webpages, "data/objs/urlContent/9_11_memorial_2011.dat.dl2");
	}
	
	public static Webpage urlResolution(Webpage p) {
		if (p.content == null) {
			return p;
		}
		//analyze the content to find the redirections. If there is no redirections, return the original
		String newContent = p.content.toLowerCase();
		while (newContent != null && newContent.contains("http-equiv=\"refresh\" content=\"0;url=")) {
			// redirection urls
			int start = newContent.indexOf("content=\"0;url=");
			int end = newContent.indexOf("\"", start + 15);
			String tempURL = p.content.substring(start + 15, end);
			tempURL = URLTools.getURL(tempURL, p.realURL);
			Webpage tempWP = d.downURL(tempURL, "IE8.0");
			p.realURL = tempWP.realURL;
			p.content = tempWP.content;
			if (tempWP.content != null) {
				newContent = tempWP.content.toLowerCase();
			} else {
				newContent = null;
			}
			// System.out.println(contents);
		}

		return p;
	}
}
